{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "①将同age、gender、education的搜索内容全部合并\n",
    "②在合并内容中，随机选取一定长度的搜索内容，作为样本的重采样\n",
    "③最终选取出20000条左右样本时候结果比较理想"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "np.random.seed(1)\n",
    "\n",
    "train = pd.read_table('E:/BaiduYunDownload/sougou2/user_tag_query.10W.TRAIN',encoding = 'gb18030',header = None,sep = '\\n',names = ['col'])\n",
    "train['ID'] = train['col'].apply(lambda x:x.split('\\t')[0])\n",
    "train['age'] = train['col'].apply(lambda x:x.split('\\t')[1])\n",
    "train['Gender'] = train['col'].apply(lambda x:x.split('\\t')[2])\n",
    "train['Education'] =  train['col'].apply(lambda x:x.split('\\t')[3])\n",
    "\n",
    "train['Query'] = train['col'].apply(lambda x:','.join(x.split('\\t')[4:-1]))\n",
    "\n",
    "train = train.drop(list(train[(train['age']=='0')|(train['Gender']=='0')|(train['Education']=='0')].index),axis = 0)\n",
    "\n",
    "train_new = train.groupby(['age','Gender','Education'],as_index = False).apply(lambda x:x['Query'].str.cat(sep=','))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame(train_new)\n",
    "c = train.groupby(['age','Gender','Education'],as_index = False).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def random_query(data,age,gender,edu):\n",
    "    L = []\n",
    "    df = pd.DataFrame()\n",
    "    count1 = int(c[(c['age']==str(age))&(c['Gender']==str(gender))&(c['Education']==str(edu))]['ID'])\n",
    "    count = count1/50 #if count1>100 else count1\n",
    "    query = (data.loc[str(age),str(gender),str(edu)].values[0]).split(',')\n",
    "    for i in range(count):\n",
    "        L.append(random.sample(query,random.randint(18,len(query))))\n",
    "    df['Query'] = L\n",
    "    df['age'] = age\n",
    "    df['Gender'] = gender\n",
    "    df['Education'] = edu\n",
    "    yield df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "T = pd.DataFrame()\n",
    "for age in range(1,7):\n",
    "    for gender in range(1,2):\n",
    "        for edu in range(1,7):\n",
    "            T = T.append(random_query(df,age,gender,edu).next())\n",
    "T.to_csv('E:/BaiduYunDownload/sougou2/TT/user_tag_T60.csv',encoding = 'gb18030')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
